import os.path
from paramiko import SSHClient
from scp import SCPClient
import paramiko
from datetime import datetime
import pytz
import re
import time
import subprocess
import scp
from tabulate import tabulate
import os

# SSH/SCP Directory Recursively     
def ssh_scp_files(ssh_host, ssh_user, ssh_password, ssh_port, local_volume, remote_volume):
    # logging.info("In ssh_scp_files()method, to copy the files to the server")
    ssh = SSHClient()
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    ssh.load_system_host_keys()
    ssh.connect(ssh_host, username=ssh_user, password=ssh_password, port=ssh_port, look_for_keys=False)

    with SCPClient(ssh.get_transport()) as scp:
        scp.get(remote_volume, local_volume, recursive=True)

def send_message(model_name,  checked_models, user_id, wandb_link="暂无", ssh_host="", ssh_user="", ssh_password="", ssh_port=0, local_volume="", remote_volume="", update_time_bound=3600, is_test=False, prev_iters={}):
    try:
        if is_test:
            test_info = '【测试，请忽略】'
        else:
            test_info = ''
        loss = -1

        if ssh_host:
            filename = local_volume + "/" + remote_volume.rsplit('/')[-1]
        else:
            filename = remote_volume
        time_print = datetime.now(pytz.timezone('Asia/Shanghai')).strftime('%Y-%m-%d %H:%M:%S')

        if model_name not in checked_models:
            return checked_models, prev_iters

        # 检查是否能连接上外部服务器
        connected = True
        for i in range(3):
            if ssh_host:
                try:
                    ssh_scp_files(ssh_host, ssh_user, ssh_password, ssh_port, local_volume, remote_volume)
                # except scp.SCPException:
                except:
                    connected = False
            if connected:
                break 

        if not (os.path.exists(filename) and connected):
            curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_warning' \
                    --header 'Content-Type: application/json' \
                    --data-raw ' {{ 
                        "text": " {} 警告： 未能连接{}模型训练的远程服务器; 当前时间为{}",
                        "user_list": ["gb8bgd55"]
                    }}'""".format(test_info, model_name, str(time_print))
            subprocess.run(curl_command, shell=True)
            return checked_models, prev_iters
        
        # 检查日志文件是否在规定时间内更新
        # Get the modification time of the file and current time
        file_modified_time = os.path.getmtime(filename)
        
        current_time = time.time()
        
        # Calculate the time difference in seconds
        time_diff = current_time - file_modified_time
        if time_diff > update_time_bound:    # 模型日志一小时内未更新
            curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_warning' \
                    --header 'Content-Type: application/json' \
                    --data-raw ' {{ 
                        "text": " {} 警告： {}模型日志半小时内未更新，怀疑训练出现问题; 当前时间为 {} ",
                        "user_list": ["{}"]
                    }}'""".format(test_info, model_name, str(time_print), user_id)
            subprocess.run(curl_command, shell=True)
            checked_models.remove(model_name) 
            return checked_models, prev_iters
        
        # 打印日志信息
        with open(filename, 'r') as f:
            last_line = None
            lines = [line.rstrip('\n') for line in f.readlines()]
        # import pdb;pdb.set_trace()
        for i in reversed(range(len(lines))):
            try:
                tmp = [e.strip() for e in lines[i].split('|')]
                loss_str = [e for e in tmp if e.startswith('loss')][0]
            except IndexError:
                continue

            pattern = r'iteration\s(.*?)(?=\|)'
            match = re.search(pattern, lines[i])
            if match:
                content = match.group(0)
            
            loss = float(loss_str.split()[1])
            break # Once find a info line, exit


        cur_iter = content.split()[1][:-1]
        if cur_iter == prev_iters[model_name]:
            curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_warning' \
                    --header 'Content-Type: application/json' \
                    --data-raw ' {{ 
                        "text": " {} 警告： {}模型iteration半小时内未更新，怀疑训练出现问题; 当前时间为 {} ",
                        "user_list": ["{}"]
                    }}'""".format(test_info, model_name, str(time_print), user_id)
            subprocess.run(curl_command, shell=True)
            checked_models.remove(model_name) 
            return checked_models, prev_iters
        prev_iters[model_name] = cur_iter

        if loss != -1:   # 检查是否在log里找到loss信息
            if loss <= 0:   # 检测loss是否为0，
                curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_warning' \
                        --header 'Content-Type: application/json' \
                        --data-raw ' {{ 
                            "text": " {} 警告： {}模型loss为0或者为负数，怀疑训练出现问题; 当前时间为{} ",
                            "user_list": ["{}"]
                        }}'""".format(test_info, model_name, str(time_print), user_id)
                checked_models.remove(model_name) 
            if loss > 3:   # 检测loss是否大于3
                curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_warning' \
                        --header 'Content-Type: application/json' \
                        --data-raw ' {{ 
                            "text": " {} 警告： {}模型loss大于3，怀疑训练出现问题; 当前时间为{} ",
                            "user_list": ["{}"]
                        }}'""".format(test_info, model_name, str(time_print), user_id)
                checked_models.remove(model_name) 
            else:
                curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_message' \
                        --header 'Content-Type: application/json' \
                        --data-raw ' {{ 
                            "text": " {} ---------{}---------\\n 模型训练正在进行中; 当前时间为{}  \\n 进程：{} \\n 当前loss为{} \\n wandb链接:{}",
                            "user_list": []
                        }}'""".format(test_info, model_name, str(time_print), content, str(loss), wandb_link)
        else:
            curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_message' \
                    --header 'Content-Type: application/json' \
                    --data-raw ' {{ 
                        "text": " {} ---------{}---------\\n 模型训练正在进行中; 当前时间为{}  \\n 进程： 尚未有任何loss信息",
                        "user_list": []
                    }}'""".format(test_info, model_name, str(time_print))
        subprocess.run(curl_command, shell=True)
    except:
        curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_warning' \
                --header 'Content-Type: application/json' \
                --data-raw ' {{ 
                    "text": " {} 警告： {} 出现为止异常; 当前时间为{}",
                    "user_list": ["{}"]
                }}'""".format(test_info, model_name, str(time_print), user_id)  
        # curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_message' \
        #         --header 'Content-Type: application/json' \
        #         --data-raw '{
        #             "text": " """ + test_info + """ """ + model_name + """\\n 出现未知异常; 当前时间为""" + str(time_print) + """",
        #             "user_list": ["9gdf9d4e"]
        #         }'"""   
        subprocess.run(curl_command, shell=True) 
        return checked_models
    return checked_models, prev_iters


def send_message_calc_avg(model_name, wandb_link, operation_link, checked_models, user_id, ssh_host="", ssh_user="", ssh_password="", ssh_port=0, local_volume="", remote_volume="", update_time_bound=1800, is_test=False,avg_range=50,abnormal_threshold=0.5):
    #try:
    if True:
        if is_test:
            test_info = '【测试，请忽略】'
        else:
            test_info = ''

        if ssh_host:
            filename = local_volume + "/" + remote_volume.rsplit('/')[-1]
        else:
            filename = remote_volume
        time_print = datetime.now(pytz.timezone('Asia/Shanghai')).strftime('%Y-%m-%d %H:%M:%S')

        if model_name not in checked_models:
            return checked_models
        
        # 检查是否能连接上外网服务
        connected = True
        if ssh_host:
            try:
                ssh_scp_files(ssh_host, ssh_user, ssh_password, ssh_port, local_volume, remote_volume)
            except scp.SCPException:
                connected = False
        
        if not (os.path.exists(filename) and connected):
            curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_warning' \
                    --header 'Content-Type: application/json' \
                    --data-raw ' {{ 
                        "text": " {} 警告： 未能连接{}模型训练的远程服务器; 当前时间为{}",
                        "user_list": ["{}"]
                    }}'""".format(test_info, model_name, str(time_print), user_id)
            subprocess.run(curl_command, shell=True)
            return checked_models
        
        # 检查日志文件是否在规定时间内更新
        # Get the modification time of the file and current time
        file_modified_time = os.path.getmtime(filename)
        import time
        current_time = time.time()

        # Calculate the time difference in seconds
        #time_diff = current_time - file_modified_time
        #if time_diff > update_time_bound:    # 模型日志一小时内未更新
        #    curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_warning' \
        #            --header 'Content-Type: application/json' \
        #            --data-raw ' {{ 
        #                "text": " {} 警告： {}模型日志半小时内未更新，怀疑训练出现问题; 当前时间为 {} ",
        #                "user_list": ["{}"]
        #            }}'""".format(test_info, model_name, str(time_print), user_id)
        #    subprocess.run(curl_command, shell=True)
        #    checked_models.remove(model_name) 
        #    return checked_models
        
        # 打印日志信息
        with open(filename, 'r') as f:
            last_line = None
            lines = []
            #for li, line in enumerate(f.readlines()):
            li = 0
            ## templ solution for unicode
            while True:
                try:
                    line = next(f)
                    lines.append(line.rstrip('\n'))
                    li += 1
                except Exception as e:
                    if li < 3000:
                        continue
                    else:
                        break

        losses = []
        count = 0
        latest = True
        content = ''
        process_rank = '-1'
        iterations = []
        # saved = []
        for i in reversed(range(len(lines))):
            try:
                tmp = [e.strip() for e in lines[i].split('|')]
                loss_str = [e for e in tmp if e.startswith('lm loss')][0]
                print(loss_str)
            except IndexError:
                continue

            # Ensure only deal with rank 0
            # Get current rank 
            #pattern = '\[Rank \d+\]'
            #match = re.search(pattern, lines[i])
            #if match:
            #    res = match.group(0)
            #    cur_rank = res.split()[-1][:-1]
            #if cur_rank != '0':    
            #    continue

            if latest:
                pattern = r'iteration\s(.*?)(?=\|)'
                match = re.search(pattern, lines[i])
                if match:
                    content = match.group(0)

                # Get latest iteration and time
                latest_iteration = content.split()[1][:-1]
                #iterations.append(latest_iteration)
                try:
                    iterations.append(latest_iteration)
                except ValueError:
                    continue
                #latest_time = lines[i].split(']')[0][1:].split(',')[0]
                import time
                latest_time = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
                latest_time = datetime.strptime(latest_time, '%Y-%m-%d %H:%M:%S')
                latest = False

            pattern = r'iteration\s(.*?)(?=\|)'
            match = re.search(pattern, lines[i])
            if match:
                content_ = match.group(0)
            iteration = content_.split()[1][:-1]
            all_iteration = content_.split()[-1]
            consumed_tokens = eval(re.search(r'consumed samples:\s(.*?)(\|)', lines[i]).group(0).split()[2])*4096/1024/1024/1024
            time_per_step=eval(re.search(r'elapsed time(.*?)(\|)', lines[i]).group(0).split()[-2])/1000.0
            time_to_finish = (eval(all_iteration) - eval(iteration)) * time_per_step / 3600.0
            try:
                #import  pdb;pdb.set_trace()
                loss = float(loss_str.split(" ")[-1])
            except ValueError:
                continue
            losses.append(loss)
            iterations.append(iteration)
            count += 1
            if count == 50:
                break

        iterations = [int(e) for e in iterations]
        iterations.sort()
        ''' 
        try:
            time_interval = (latest_time - foremost_time).total_seconds()
            # 检查50个iteration的时间是否超过预计值
            # if time_interval > 49*20*2:
            if time_interval > (iterations[-1]-iterations[0]+1)*20*2:
                curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_warning' \
                        --header 'Content-Type: application/json' \
                        --data-raw ' {{ 
                            "text": " {} 警告： {}模型训练降速，怀疑出现问题 当前时间为{} ",
                            "user_list": ["{}"]
                        }}'""".format(test_info, model_name, str(time_print), user_id)
                subprocess.run(curl_command, shell=True)
                checked_models.remove(model_name) 
                return checked_models
        except NameError:
            pass
        '''
        
        if len(losses) != 0:   # 检查是否在log里找到loss信息
            loss = float("{:.3f}".format(sum(losses)/len(losses)))
            if eval(iteration) > 2000 and (loss <= 0 or max(losses) > min(losses) + abnormal_threshold):   # 检测loss是否为0，
                curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_warning' \
                        --header 'Content-Type: application/json' \
                        --data-raw ' {{ 
                            "text": " {} 警告： {}模型loss出现0、负数或者异常峰值，怀疑训练出现问题; 当前时间为{} ",
                            "user_list": ["{}"]
                        }}'""".format(test_info, model_name, str(time_print), user_id)
                checked_models.remove(model_name) 
            else:
                curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_message' \
                        --header 'Content-Type: application/json' \
                        --data-raw ' {{ 
                            "text": " {} ---------{}---------\\n 模型训练正在进行中; 当前时间为{}  \\n 进程：{} \\n 过去{}步平均loss为{} \\n 已消耗的tokens量: {} B \\n 当前训练速度: {} s/step \\n 预计还需训练时间: {} 小时 \\n wandb链接:{} \\n 训练记录文档:{}",
                            "user_list": []
                        }}'""".format(test_info, model_name, str(time_print), content, str(len(losses)), str(loss), "%.2f" % consumed_tokens, "%.2f" % time_per_step, "%.2f" % time_to_finish, wandb_link, operation_link)
        else:
            curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_message' \
                    --header 'Content-Type: application/json' \
                    --data-raw ' {{ 
                        "text": " {} ---------{}---------\\n 模型训练正在进行中; 当前时间为{}  \\n 进程： 尚未有任何loss信息",
                        "user_list": []
                    }}'""".format(test_info, model_name, str(time_print))
        
        subprocess.run(curl_command, shell=True)
    #except Exception as e:
    #    print(e)
    #if False:
        '''
        if not is_test:
            curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_message' \
                    --header 'Content-Type: application/json' \
                    --data-raw '{
                        "text": " """ + test_info + """ """ + model_name + """\\n 出现未知异常; 当前时间为""" + str(time_print) + """",
                        "user_list": ["gb8bgd55"]
                    }'"""   
            subprocess.run(curl_command, shell=True) 
        return checked_models
        '''
        return 1
    return checked_models


def send_message_val_old(model_name, wandb_link="暂无", checked_models=[], user_id=0, ssh_host="", ssh_user="", ssh_password="", ssh_port=0, local_volume="",
                     remote_volume_wiki="", remote_volume_domain="", remote_volume_code="", wiki_filename="", domain_filename="", code_filename="", update_time_bound=3600*5, is_test=False,num_lines=3):
                     
    try:
        if model_name not in checked_models:
            return checked_models
        # try:
        if is_test:
            test_info = '【测试，请忽略】\\n'
        else:
            test_info = ''
        loss = -1
            

        time_print = datetime.now(pytz.timezone('Asia/Shanghai')).strftime('%Y-%m-%d %H:%M:%S')

        if model_name not in checked_models:
            return checked_models

        if ssh_host:
            wiki_filename = local_volume + "/" + remote_volume_wiki.rsplit('/')[-1]
            domain_filename = local_volume + "/" + remote_volume_domain.rsplit('/')[-1]
            connected = True
            for i in range(3):
                if ssh_host:
                    try:
                        ssh_scp_files(ssh_host, ssh_user, ssh_password, ssh_port, local_volume, remote_volume_wiki)
                        ssh_scp_files(ssh_host, ssh_user, ssh_password, ssh_port, local_volume, remote_volume_domain)
                    # except scp.SCPException:
                    except:
                        connected = False
                if connected:
                    break 
            if not (os.path.exists(wiki_filename) and os.path.exists(domain_filename) and connected):
                curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_warning' \
                        --header 'Content-Type: application/json' \
                        --data-raw ' {{ 
                            "text": " {} 警告： 未能连接{}模型训练的远程服务器; 当前时间为{}",
                            "user_list": ["gb8bgd55"]
                        }}'""".format(test_info, model_name, str(time_print))
                subprocess.run(curl_command, shell=True)
                return checked_models, prev_iters
        # 检查日志文件是否在规定时间内更新
        # Get the modification time of the file and current time
        file_modified_time_wiki = os.path.getmtime(wiki_filename)
        file_modified_time_domain = os.path.getmtime(domain_filename)
        current_time = time.time()
        
        # Calculate the time difference in seconds
        # if (current_time - file_modified_time_wiki > update_time_bound) or (current_time - file_modified_time_domain > update_time_bound):    # 模型日志一小时内未更新
        #     curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_warning' \
        #             --header 'Content-Type: application/json' \
        #             --data-raw ' {{ 
        #                 "text": " {} 警告： {}日志五小时内未更新，怀疑训练出现问题; 当前时间为 {} ",
        #                 "user_list": ["{}"]
        #             }}'""".format(test_info, model_name, str(time_print), user_id)
        #     subprocess.run(curl_command, shell=True)
        #     checked_models.remove(model_name) 
        #     return checked_models
        
        # 打印日志信息
        wiki_lines, domain_lines = [],[]
        with open(wiki_filename, 'r') as f:
            wiki_lines = [line.rstrip('\n').split() for line in f.readlines()][-num_lines:]
            wiki_lines = [[int(e[0]), "{:.3f}".format(float(e[1])), "     " if len(e)==2 else "{:.3f}".format(float(e[2]))] for e in wiki_lines]

        with open(domain_filename, 'r') as f:
            domain_lines = [line.rstrip('\n').split() for line in f.readlines()][-num_lines:]
            domain_lines = [[int(e[0]), "{:.3f}".format(float(e[1])), "     " if len(e)==2 else "{:.3f}".format(float(e[2]))] for e in domain_lines]        

        # 只考虑了领先了一步的情况
        if wiki_lines[-1][0] >  domain_lines[-1][0]:
            domain_lines = domain_lines[1:]
            domain_lines.append(([wiki_lines[-1][0],"暂无","暂无"]))
        assert [e[0] for e in wiki_lines] == [e[0] for e in domain_lines]   
        
        
        content = [ [wiki_lines[i][0]] + [wiki_lines[i][1],wiki_lines[i][2],domain_lines[i][1],domain_lines[i][2]] for i in range(len(wiki_lines))]
        # first_line = [wiki_lines[-1][0]] + wiki_lines[-1][1]+domain_lines[-1][1]
        # wiki_info = ['wiki'] + [e[1] for e in wiki_lines]
        # domain_info = ['domain'] + [e[1] for e in domain_lines]
        header_info = ['iter \\\ metrics'] + ["wiki loss", "wiki acc", "domain loss", "domain acc"]
        info = tabulate(content, headers=header_info, tablefmt='orgtbl')
        info = "\\n ".join(info.split('\n'))
        # info = info[0]+' '+info[1:]
        curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_message' \
            --header 'Content-Type: application/json' \
            --data-raw ' {{
                "text": " {} {} loss评测结果: \\n当前时间:{} \\n{}",
                "user_list": []
            }}'""".format(test_info, model_name,str(time_print), info)   
        subprocess.run(curl_command, shell=True)
    except:
        curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_warning' \
                --header 'Content-Type: application/json' \
                --data-raw ' {{ 
                    "text": " {} 警告： {} 出现为止异常; 当前时间为{}",
                    "user_list": ["{}"]
                }}'""".format(test_info, model_name, str(time_print), user_id)  
        # curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_message' \
        #         --header 'Content-Type: application/json' \
        #         --data-raw '{
        #             "text": " """ + test_info + """ """ + model_name + """\\n 出现未知异常; 当前时间为""" + str(time_print) + """",
        #             "user_list": ["9gdf9d4e"]
        #         }'"""   
        subprocess.run(curl_command, shell=True) 
    return checked_models

def send_message_val(model_name, wandb_link="暂无", checked_models=[], user_id=0, ssh_host="", ssh_user="", ssh_password="", ssh_port=0, local_volume="",
                     remote_volume="", filenames=[], dataset_names=[], update_time_bound=3600*5, is_test=False, num_lines=4):
                     
    try:
        if model_name not in checked_models:
            return checked_models
        # try:
        if is_test:
            test_info = '【测试，请忽略】\\n'
        else:
            test_info = ''
        loss = -1
        

        
        time_print = datetime.now(pytz.timezone('Asia/Shanghai')).strftime('%Y-%m-%d %H:%M:%S')

        if model_name not in checked_models:
            return checked_models
        

        filenames = [os.path.join(local_volume,filename) for filename in filenames]
        # 检查日志文件是否在规定时间内更新
        # Get the modification time of the file and current time
        if not ssh_host:
            file_modified_time = [os.path.getmtime(filename) for filename in filenames]
            current_time = time.time()

            # Calculate the time difference in seconds
            if True in [((current_time - modtime) > update_time_bound) for modtime in file_modified_time]:    # 模型日志一小时内未更新
                curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_warning' \
                        --header 'Content-Type: application/json' \
                        --data-raw ' {{ 
                            "text": " {} 警告： {}日志五小时内未更新，怀疑训练出现问题; 当前时间为 {} ",
                            "user_list": ["{}"]
                        }}'""".format(test_info, model_name, str(time_print), user_id)
                subprocess.run(curl_command, shell=True)
                checked_models.remove(model_name) 
                return checked_models
        
        # 打印日志信息
        lines = [[] for i in range(len(filenames))]
        for i in range(len(lines)):
            with open(filenames[i], 'r') as f:
                tmp = f.readlines()
                if ',' in tmp[-1]:
                    lines[i] = [line.rstrip('\n').split(',') for line in tmp][-num_lines:]
                else:
                    lines[i] = [line.rstrip('\n').split() for line in tmp][-num_lines:]
                

                lines[i] = [[int(e[0]), "{:.3f}".format(float(e[1])), "     " if len(e)==2 else "{:.3f}".format(float(e[2]))] for e in lines[i]]

        # 只考虑了领先了一步的情况
        for i in range(len(filenames)-1):
            if lines[0][-1][0] > lines[i+1][-1][0]:
                lines[i+1] = lines[i+1][1:]
                lines[i+1].append(([lines[0][-1][0],"暂无","暂无"]))

        if len(lines) == 3:
            content = [ [lines[0][i][0]] + [lines[0][i][1],lines[0][i][2],lines[1][i][1],lines[1][i][2],lines[2][i][1],lines[2][i][2]] for i in range(len(lines[0]))]
        elif len(lines) == 2:
            content = [ [lines[0][i][0]] + [lines[0][i][1],lines[0][i][2],lines[1][i][1],lines[1][i][2]] for i in range(len(lines[0]))]
        if len(lines) == 5:
            content = [ [lines[0][i][0]] + [lines[0][i][1],lines[0][i][2],lines[1][i][1],lines[1][i][2],lines[2][i][1],lines[2][i][2],lines[3][i][1],lines[3][i][2],lines[4][i][1],lines[4][i][2]] for i in range(len(lines[0]))]

        tmp = [filename.split('/')[-1].split('_')[0] for filename in filenames]
        headers = []
        for name in tmp:
            headers += [name+' loss', name+' acc']
        
        header_info = ['iter \\\ metrics'] + headers
        info = tabulate(content, headers=header_info, tablefmt='orgtbl')
        info = "\\n ".join(info.split('\n'))
        # info = info[0]+' '+info[1:]
        curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_message' \
            --header 'Content-Type: application/json' \
            --data-raw ' {{
                "text": " {} {} loss评测结果: \\n当前时间:{} \\n{}",
                "user_list": []
            }}'""".format(test_info, model_name,str(time_print), info)   
        subprocess.run(curl_command, shell=True)
    except:
        curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_message' \
                --header 'Content-Type: application/json' \
                --data-raw '{
                    "text": " """ + test_info + """ """ + model_name + """\\n 出现未知异常; 当前时间为""" + str(time_print) + """",
                    "user_list": ["gb8bgd55"]
                }'"""   
        subprocess.run(curl_command, shell=True) 
    return checked_models


def send_message_val_long(model_name, wandb_link="暂无", checked_models=[], user_id=0, ssh_host="", ssh_user="", ssh_password="", ssh_port=0, local_volume="",
                     remote_volume="", filenames=[], dataset_names=[], update_time_bound=3600*8, is_test=False, num_lines=4):
                     
    try:
        if model_name not in checked_models:
            return checked_models
        # try:
        if is_test:
            test_info = '【测试，请忽略】\\n'
        else:
            test_info = ''
        loss = -1
        

        
        time_print = datetime.now(pytz.timezone('Asia/Shanghai')).strftime('%Y-%m-%d %H:%M:%S')

        if model_name not in checked_models:
            return checked_models
        

        filenames = [os.path.join(local_volume,filename) for filename in filenames]
        # 检查日志文件是否在规定时间内更新
        # Get the modification time of the file and current time
        # if not ssh_host:
        #     file_modified_time = [os.path.getmtime(filename) for filename in filenames]
        #     current_time = time.time()

        #     # Calculate the time difference in seconds
        #     if True in [((current_time - modtime) > update_time_bound) for modtime in file_modified_time]:    # 模型日志一小时内未更新
        #         curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_warning' \
        #                 --header 'Content-Type: application/json' \
        #                 --data-raw ' {{ 
        #                     "text": " {} 警告： {}日志五小时内未更新，怀疑训练出现问题; 当前时间为 {} ",
        #                     "user_list": ["{}"]
        #                 }}'""".format(test_info, model_name, str(time_print), user_id)
        #         subprocess.run(curl_command, shell=True)
        #         checked_models.remove(model_name) 
        #         return checked_models
        
        # 打印日志信息
        lines = [[] for i in range(len(filenames))]
        for i in range(len(lines)):
            with open(filenames[i], 'r') as f:
                tmp = f.readlines()
                if ',' in tmp[-1]:
                    lines[i] = [line.rstrip('\n').split(',') for line in tmp]

                else:
                    lines[i] = [line.rstrip('\n').split() for line in tmp]


                # To prevent the catse that two lines have the same iteration 
                lines[i] = [lines[i][j] for j in range(len(lines[i])-1) if lines[i][j][0] != lines[i][j+1][0]]+[lines[i][-1]]
                lines[i] = lines[i][-num_lines:]
                # omit "outputs-"
                lines[i] = [[int(re.findall('\d+', e[0])[-1]), "{:.3f}".format(float(e[1])), "     " if len(e)==2 else "{:.3f}".format(float(e[2]))] for e in lines[i]]

        # 只考虑了领先了一步的情况
        for i in range(len(filenames)-1):
            if lines[0][-1][0] > lines[i+1][-1][0]:
                lines[i+1] = lines[i+1][1:]
                lines[i+1].append(([lines[0][-1][0],"暂无","暂无"]))
                
        content = [ [lines[0][i][0]] + [lines[j][i][1] for j in range(len(lines))]  for i in range(len(lines[0]))]
        tmp = [filename.split('/')[-1].split('.')[0] for filename in filenames]
        headers = []
        for name in tmp:
            headers += [name+' loss']
        
        header_info = ['iter \\\ metrics'] + headers
        info = tabulate(content, headers=header_info, tablefmt='orgtbl')
        info = "\\n ".join(info.split('\n'))
        # info = info[0]+' '+info[1:]
        curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_message' \
            --header 'Content-Type: application/json' \
            --data-raw ' {{
                "text": " {}{}wandb链接: {} \\n{} loss评测结果: \\n当前时间:{} \\n{}",
                "user_list": []
            }}'""".format(test_info, model_name, wandb_link, model_name,str(time_print), info)   
        subprocess.run(curl_command, shell=True)


        content = [ [lines[0][i][0]] + [lines[j][i][2] for j in range(len(lines))]  for i in range(len(lines[0]))]

        tmp = [filename.split('/')[-1].split('.')[0] for filename in filenames]
        headers = []
        for name in tmp:
            headers += [name+' acc']
        
        header_info = ['iter \\\ metrics'] + headers
        info = tabulate(content, headers=header_info, tablefmt='orgtbl')
        info = "\\n ".join(info.split('\n'))
        # info = info[0]+' '+info[1:]
        curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_message' \
            --header 'Content-Type: application/json' \
            --data-raw ' {{
                "text": " {} {} acc评测结果: \\n当前时间:{} \\n{}",
                "user_list": []
            }}'""".format(test_info, model_name,str(time_print), info)   
        subprocess.run(curl_command, shell=True)

    except:   
        curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_warning' \
                --header 'Content-Type: application/json' \
                --data-raw ' {{ 
                    "text": " {} 警告： {} 出现为止异常; 当前时间为{}",
                    "user_list": ["{}"]
                }}'""".format(test_info, model_name, str(time_print), user_id)  
        subprocess.run(curl_command, shell=True) 
    return checked_models


def send_message_val_single(model_name, wandb_link="暂无", checked_models=[], user_id=0, ssh_host="", ssh_user="", ssh_password="", ssh_port=0, local_volume="",
                     remote_volume="", filename="", update_time_bound=3600*5, is_test=False,num_lines=3):
    try:
        if model_name not in checked_models:
            return checked_models
        # try:
        if is_test:
            test_info = '【测试，请忽略】\\n'
        else:
            test_info = ''
        loss = -1
            

        time_print = datetime.now(pytz.timezone('Asia/Shanghai')).strftime('%Y-%m-%d %H:%M:%S')

        if model_name not in checked_models:
            return checked_models

        if ssh_host:
            filename = local_volume + "/" + remote_volume.rsplit('/')[-1]
            connected = True
            for i in range(3):
                if ssh_host:
                    try:
                        ssh_scp_files(ssh_host, ssh_user, ssh_password, ssh_port, local_volume, remote_volume)
                    # except scp.SCPException:
                    except:
                        connected = False
                if connected:
                    break 
            if not (os.path.exists(filename) and connected):
                curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_warning' \
                        --header 'Content-Type: application/json' \
                        --data-raw ' {{ 
                            "text": " {} 警告： 未能连接{}模型训练的远程服务器; 当前时间为{}",
                            "user_list": ["gb8bgd55"]
                        }}'""".format(test_info, model_name, str(time_print))
                subprocess.run(curl_command, shell=True)
                return checked_models, prev_iters

        # 打印日志信息
        lines = []
        with open(filename, 'r') as f:
            lines = [line.rstrip('\n').split(',') for line in f.readlines()][-num_lines:]
            lines = [[int(e[0]), "{:.4f}".format(float(e[1])), "     " if len(e)==2 else "{:.4f}".format(float(e[2]))] for e in lines]



        content = [ [lines[i][0]] + [lines[i][1],lines[i][2]] for i in range(len(lines))]

        header_info = ['iter \\\ metrics'] + ["0.8.4 loss", "0.8.4 acc"]
        info = tabulate(content, headers=header_info, tablefmt='orgtbl')
        info = "\\n ".join(info.split('\n'))
        # info = info[0]+' '+info[1:]
        curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_message' \
            --header 'Content-Type: application/json' \
            --data-raw ' {{
                "text": " {} {} loss评测结果: \\n当前时间:{} \\n{}",
                "user_list": []
            }}'""".format(test_info, model_name,str(time_print), info)   
        subprocess.run(curl_command, shell=True)
    except:
        curl_command = """curl --location --request POST 'https://flagopen.baai.ac.cn/warning/send_message' \
                --header 'Content-Type: application/json' \
                --data-raw '{
                    "text": " """ + test_info + """ """ + model_name + """\\n 出现未知异常; 当前时间为""" + str(time_print) + """",
                    "user_list": ["gb8bgd55"]
                }'"""   
        subprocess.run(curl_command, shell=True) 
    return checked_models
